# -*- coding: utf-8 -*-

import os
import sys

if __name__ == "__main__":
    currentdir = os.path.dirname( os.path.abspath( sys.argv[0] ) )
    basedir = os.path.abspath( os.path.join( currentdir, ".." ) )
    pythondir = basedir
    sys.path.insert( 0, pythondir )
    pass

from logger import get_file_logger
import traceback
import mimetypes
from time import time
from picklerpc.msghandler import RequestHandler
from picklerpc.sendmsg import sendmsg
from picklerpc import Server
from time import sleep
from datetime import timedelta
from multiprocessing import cpu_count
from service import default_request_timeout, default_niceness_level, Py34ServiceManager as ServiceManager

__all__ = ['ProcessArchiveHandler', 'ProcessArchiveServer', 'ProcessArchiveRequest', 'ProcessArchiveManager']

#---------------------------------------------

class ProcessArchiveHandler(RequestHandler):
    """
    Do some tasks, maybe in subprocess
    """
    
    def _request_handle(self, data):
        """
        @param data: simple object data
        @type: dict
        
        @return: python simple object data, jsonable via json.dumps
        @rtype: mixed
        """
        result = False
        
        if not (type(data) is dict):
            return result

        if data["task"] == "ping":
            return self.server.ping()
        if data["task"] == "quit":
            return self.server.quit()
        if data["task"] == "process":
            data = data["data"]
            return self.server.process(
                data["file_id"],
                data["file_path"],
                data["mime"],
                data["fsize"],
            )
        return result
    
    pass

#---------------------------------------------

class ProcessArchiveServer(Server):
    
    queue = None
    stop_event = None
    task_service_address = None
    _worker_thread = None


    img_manager = None
    img_req = None

    _img_req_lock = None

    log = None

    def server_activate(self):
        try:
            # Python 3
            from queue import Queue
        except ImportError:
            # Python 2
            from Queue import Queue

        from threading import Thread, Event, RLock

        self.log = get_file_logger("ProcessArchiveServer")

        self._img_req_lock = RLock()
        self.queue = Queue()
        self.stop_event = Event()
        self._worker_thread = Thread(target=self._worker)
        self._worker_thread.start()

        return Server.server_activate(self)


    def ping(self):
        return "pong"


    def remove_temp_directory(self, tmp_dir):

        all_files = []
        all_dirs = []

        for root, dirs, files in os.walk(tmp_dir):

            for f in files:

                full_path = os.path.join(root, f)

                if os.path.isfile(full_path):
                    all_files.append(full_path)

            for d in dirs:

                full_path = os.path.join(root, d)

                if os.path.isdir(full_path):
                    all_dirs.append(full_path)

        all_dirs.append(tmp_dir)

        all_files.sort()
        all_dirs.sort()
        all_dirs.reverse()

        for path in all_files:
            os.unlink(path)

        for path in all_dirs:
            try:
                os.rmdir(path)
            except:
                pass

        return True


    def getImageRequest(self, number):
        self._img_req_lock.acquire()
        self.img_manager.start_service(number)
        self._img_req_lock.release()
        if not self.img_req:
            self.img_req = self.img_manager.get_request()
        return self.img_req


    def getExtractedFiles(self, into_dir):
        all_files = []

        for root, dirs, files in os.walk(into_dir):

            if self.stop_event.isSet():
                break

            for f in files:

                if self.stop_event.isSet():
                    break

                full_path = os.path.join(root, f)
                all_files.append(full_path)

                pass

            pass
        return all_files


    def _worker ( self ):

        self.log.info("START: %s" % (self.server_address,))

        from archive import which
        from subprocess import Popen
        from service.task import TaskManager
        import tempfile
        from service.image_vector import ImageVectorManager
        from service.statistics import StatisticsDirectRequest
        from project.query import Project

        mimetypes.init()

        bin_python = which("python3")

        task_manager = TaskManager(self.task_service_address)

        self.img_manager = ImageVectorManager( self.task_service_address )

        stats_req = task_manager.get_request().getStatistics()
        while not stats_req.ping():
            sleep(0.1)
        addr = stats_req.get_address()
        stats_req = StatisticsDirectRequest(addr)


        db_req = Project()


        temp_directory = db_req.get_option("temp_directory")
        if not os.path.isdir(temp_directory):
            os.makedirs(temp_directory, 0o700)

        max_number = int(cpu_count())

        while not self.stop_event.isSet():

            try:
                data = self.queue.get_nowait()
            except Exception:
                data = None

            if data is None:
                sleep(0.1)
                continue


            file_id, file_path, mime, fsize = data

            tmp_dir = tempfile.mkdtemp(dir = temp_directory, prefix="arch-")

            try:
                stats_req.set_imaging( file_path, 0, 0, time() )

                pp = Popen([
                    bin_python, "archive/extract_tool.py",
                    "--archive", file_path,
                    "--mime", mime,
                    "--into-dir", tmp_dir
                ])

                while pp.poll() is None:
                    sleep(0.1)

                    all_files = self.getExtractedFiles(tmp_dir)
                    stats_req.set_imaging( file_path, len(all_files), 0, time() )

                    if self.stop_event.isSet():
                        pp.terminate()
                        break


                all_files = self.getExtractedFiles(tmp_dir)
                stats_req.set_imaging( file_path, len(all_files), 0, time() )

                count_files = len(all_files)

                # self.log.debug("Archive: %r, files count = %s" % (file_path, count_files,))

                if count_files:

                    for n in range(max_number):
                        num = "iw-%s" % (n % max_number)
                        self.getImageRequest(num).reset_stats(num)

                    n = 0
                    for path in all_files:

                        num = "iw-%s" % (n % max_number)

                        self.getImageRequest(num).process( num, file_id, path, tmp_dir, fsize*1.0/count_files )

                        n += 1

                    while True:
                        if self.stop_event.isSet():
                            break

                        done_nf = 0
                        for n in range(max_number):
                            num = "iw-%s" % (n % max_number)
                            stats = self.getImageRequest(num).get_stats(num)
                            if stats:
                                done_nf += stats["processed"]

                        # self.log.debug("-- processed count = %s" % (done_nf,))

                        if done_nf == count_files:
                            break

                        sleep(0.1)

                else:
                    stats_req.inc("archives_processed_size", fsize)


            except Exception as e:
                stats_req.inc("errors_count", 1)
                self.log.error("%s" % (e,))
                self.log.debug("ERROR!\n%s" % (traceback.format_exc(),))
                pass
            finally:
                self.remove_temp_directory(tmp_dir)

            stats_req.inc("archives_processed", 1)

            self.queue.task_done()

        img_req = self.img_manager.get_request()
        img_req.quit()
        while img_req.ping_all():
            sleep(0.1)

        self.log.info("STOP: %s" % (self.server_address,))

        self.stop_event.clear()
        return

    def process(self, file_id, file_path, mime, fsize):
        self.queue.put_nowait((file_id, file_path, mime, fsize,))
        return True

    def quit(self):
        self.log.info("quit event")

        import signal

        self.log.debug("send SIGINT to self")
        os.kill( os.getpid(), signal.SIGINT )

        return True


    def terminate_self(self, signum, frame):
        self.log.debug("got SHUTDOWN signal")
        t_begin = time()

        # Clean Queue

        while True:
            try:
                data = self.queue.get_nowait()
            except Exception:
                data = None

            if data is None:
                break

            self.queue.task_done()


        self.stop_event.set()
        self.stop_event.wait()
        self.server_close()
        t_end = time()
        self.log.info("SHUTDOWN in %s sec" % (timedelta(seconds=t_end - t_begin),))
        return

    pass


#---------------------------------------------

class ProcessArchiveRequest(object):

    manager = None

    def __init__(self, manager):
        self.manager = manager
        pass

    def process(self, file_id, file_path, mime, fsize):
        return self.manager.sendmsg( "process", {
            "file_id" : file_id,
            "file_path" : file_path,
            "mime" : mime,
            "fsize" : fsize
        })

    def ping(self):
        return self.manager.sendmsg( "ping", {}, 0.1)

    def quit(self):
        return self.manager.sendmsg( "quit", {}, 0.1)

    def get_address(self):
        return self.manager.get_address()

    pass

class ProcessArchiveDirectRequest(object):

    address = None

    def __init__(self, service_address):
        self.address = service_address
        self.log = get_file_logger("ProcessArchiveDirectRequest")
        pass

    def sendmsg(self, task, data, timeout=default_request_timeout):
        try:
            result = sendmsg( self.address, {"task":task, "data":data}, timeout=timeout )
        except Exception as e:
            result = None
            self.log.error("SOCKET: %s" % ( e,))
            self.log.debug("FAIL SEND MESSAGE: task=%s, data=%r" % ( task, data,))
        return result

    def process(self, file_id, file_path, mime, fsize):
        return self.sendmsg( "process", {
            "file_id" : file_id,
            "file_path" : file_path,
            "mime" : mime,
            "fsize" : fsize
        })

    def ping(self):
        return self.sendmsg( "ping", {}, 0.1)

    def quit(self):
        return self.sendmsg( "quit", {}, 0.1)

    pass

#---------------------------------------------

class ProcessArchiveManager(ServiceManager):

    host = "127.0.1.7"
    port_min = 10000
    port_max = 20000
    port = None
    request = None

    address = None

    task_service_address = None

    log = None

    def __init__(self, task_service_address):
        self.task_service_address = task_service_address
        self.log = get_file_logger("ProcessArchiveManager")
        pass

    def start_service(self):

        runing = False
        if not self.port:

            import random

            self.port = random.randint(self.port_min, self.port_max)

            self.address = (self.host, self.port,)

            from subprocess import Popen

            Popen([self.getPython(), "service/process_archive.py",
                   "--host", "%s" % self.host,
                   "--port", "%s" % self.port,
                   "--task-service", "%s:%s" % self.task_service_address
            ])

            # WAIT WHILE SERVICE STARTS

            pings = 100
            t_begin = time()
            while not runing:

                pong = self.get_request().ping()

                if pong == "pong":
                    runing = True

                pings -= 1
                if pings <= 0:
                    break

            if pings <= 0:
                self.port = None
                self.log.error("FAIL TO START service on %r in %s sec" % (
                    self.address, timedelta(seconds=time() - t_begin),))
            else:
                self.log.info("STARTED service on %r in %s sec" % (
                    self.address, timedelta(seconds=time() - t_begin),))

        return runing

    def sendmsg(self, task, data, timeout=default_request_timeout):
        try:
            result = sendmsg( self.address, {"task":task, "data":data}, timeout=timeout )
        except Exception as e:
            result = None
            self.log.error("SOCKET: %s" % ( e,))
            self.log.debug("FAIL SEND MESSAGE: task=%s, data=%r" % ( task, data,))
        return result

    def quit(self):
        return self.get_request().quit()

    def get_address(self):
        return self.address

    def get_request(self):
        if not self.request:
            self.request = ProcessArchiveRequest(self)
        return self.request

    pass


#---------------------------------------------

def run_service(host, port, task_service_address):

    if hasattr(os, "nice"):
        os.nice(default_niceness_level)

    server = None
    e_code = 0
    try:
        SERVER_ADDRESS = (host, int(port),)

        server = ProcessArchiveServer(SERVER_ADDRESS, ProcessArchiveHandler, bind_and_activate=False)
        if task_service_address:
            host, port = task_service_address.split(':')
            server.task_service_address = (host, int(port),)

        import signal

        signal.signal(signal.SIGINT, server.terminate_self)
        signal.signal(signal.SIGTERM, server.terminate_self)

        server.server_bind()
        server.server_activate()
        server.serve_forever()
    except Exception as e:
        e_code = 1
        if server and server.log:
            server.log.error("run_service: ERROR=%r" % (e,))
        pass

    return e_code


#---------------------------------------------

def launch():

    from argparse import ArgumentParser

    parser = ArgumentParser(description='Run image archive processing service.')
    parser.add_argument('--host', dest='host', action='store', type=str,
        help='Listen on TCP address')
    parser.add_argument('--port', dest='port', action='store', type=int,
        help='Listen on TCP port')
    parser.add_argument('--task-service', dest='task_service', action='store', type=str,
        help='Task service listen on TCP address')

    args = parser.parse_args()
    return run_service(args.host, args.port, args.task_service)

if __name__ == "__main__":
    sys.exit(launch())
